#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Project: spd-sxmcc
"""
@author: lyndon
@time Created on 2018/11/19 16:36
@desc
"""

import re
from pyspider.libs.base_handler import *
import urllib
import urllib2
import lxml.html as HTML


import os
import hashlib
import cookielib

LOGIN_URL = 'https://www.qichacha.com/user_login'
USER_NAME = '13934578599'
PASSWORD = 'ShiHuiTao332324'

HOST = 'www.qichacha.com'
REFERER = 'https://www.qichacha.com/'
POST_URL_PREFIX = 'https://www.qichacha.com/'

# !!! Notice !!!
# Tasks that share the same account MUST share the same cookies file
COOKIES_FILE = '/tmp/pyspider.%s.%s.cookies' % (HOST, hashlib.md5(USER_NAME).hexdigest())
COOKIES_DOMAIN = HOST

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36'
HTTP_HEADERS = {
    'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Encoding' : 'gzip, deflate, sdch',
    'Accept-Language' : 'zh-CN,zh;q=0.8,en;q=0.6',
    'Connection' : 'keep-alive',
    'DNT' : '1',
    'Host' : HOST,
    'Referer' : REFERER,
    'User-Agent' : USER_AGENT,
}


def getCookies():
    cookiesJar = cookielib.MozillaCookieJar(COOKIES_FILE)
    if not os.path.isfile(COOKIES_FILE):
        cookiesJar.save()
    cookiesJar.load(COOKIES_FILE)
    cookieProcessor = urllib2.HTTPCookieProcessor(cookiesJar)
    cookieOpener = urllib2.build_opener(cookieProcessor, urllib2.HTTPHandler)
    for item in HTTP_HEADERS:
        cookieOpener.addheaders.append((item, HTTP_HEADERS[item]))
    urllib2.install_opener(cookieOpener)
    if len(cookiesJar) == 0:
        login = Login(USER_NAME, PASSWORD, LOGIN_URL, POST_URL_PREFIX)
        if login.login():
            cookiesJar.save()
        else:
            return None
    cookiesDict = {}
    for cookie in cookiesJar:
        if COOKIES_DOMAIN in cookie.domain:
            cookiesDict[cookie.name] = cookie.value
    print(cookiesDict)
    return cookiesDict


class Handler(BaseHandler):
    crawl_config = {
        # 'proxy': 'spdu:JIqnEW218W9@47.94.227.14:43218'
        'proxy': 'spdu: JIqnEW218W9 @ 47.95.230.230:43218'
    }

    @every(minutes=24 * 60)
    def on_start(self):
        cookies = getCookies()
        for page in range(1, 101):
            keywd = urllib.quote('云南 早教')
            url = 'https://www.tianyancha.com/search/p%s?key=%s' % (page, keywd)
            print(url)
            self.crawl(url, cookies=cookies, callback=self.index_page, validate_cert=False)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        cookies = getCookies()
        for each in response.doc('a[href^="http"]').items():
            if re.match("https://www\.qichacha\.com/firm_.*.html", each.attr.href):
                self.crawl(each.attr.href, cookies=cookies, callback=self.detail_page, validate_cert=False)


    @config(priority=2)
    def detail_page(self, response):
        infos = response.doc('.ntable')
        return {
            "url": response.url,
            "title": response.doc('title').text(),
            "info" : infos
        }


class Login(object):

    def __init__(self, username, password, login_url, post_url_prefix):
        self.username = username
        self.password = password
        self.login_url = login_url
        self.post_url_prefix = post_url_prefix


    def getPostData(self):
        url = self.login_url.strip()
        if not re.match(r'^http://', url):
            return None, None
        print(url)
        req = urllib2.Request(url)
        resp = urllib2.urlopen(req)
        login_page = resp.read()
        doc = HTML.fromstring (login_page)
        post_url = doc.xpath("//form[@method='post' and @id='lsform']/@action")[0]
        cookietime = doc.xpath("//input[@name='cookietime' and @id='ls_cookietime']/@value")[0]
        username = self.username
        password = self.password
        post_data = urllib.urlencode({
            'fastloginfield' : 'username',
            'username' : username,
            'password' : password,
            'quickforward' : 'no',
            'handlekey' : 'ls',
            'cookietime' : cookietime,
        })
        return post_url, post_data

    def login(self):
        post_url, post_data = self.getPostData()
        print('post_url:' + post_url)
        print('post_data:' + post_data)
        post_url = self.post_url_prefix + post_url
        req = urllib2.Request(url=post_url, data=post_data)
        resp = urllib2.urlopen(req)
        return True

